Force Plate Project

import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings("ignore")
import matplotlib.pyplot as plt
import seaborn as sns

df=pd.read_csv('allFiles.csv',index_col=0)

EDA

df.head()

	fx	fy	fz	mx	my	mz	Trial Number	Trial Type	Subject Number	Body Mass (Kg)	AP Sway (cm)	ML Sway (cm)
0	-6.47778	-11.10205	778.78125	-0.38182	-16.04395	-22.75000	1	1	Subject 001	79.2834	0.049028	-2.060136
1	-6.91699	-10.65332	777.71875	0.03605	-15.95557	-22.76465	1	1	Subject 001	79.2834	-0.004635	-2.051586
2	-6.55713	-11.10156	778.78125	-0.20785	-15.94238	-22.77148	1	1	Subject 001	79.2834	0.026689	-2.047093
3	-6.64404	-10.92285	778.40625	-0.30165	-15.87402	-22.70313	1	1	Subject 001	79.2834	0.038752	-2.039298
4	-6.74341	-10.64600	779.84375	-0.09583	-15.91504	-22.78906	1	1	Subject 001	79.2834	0.012288	-2.040799

df.tail()

	fx	fy	fz	mx	my	mz	Trial Number	Trial Type	Subject Number	Body Mass (Kg)	AP Sway (cm)	ML Sway (cm)
14995	3.85071	-7.57349	626.18750	-13.28613	30.21777	-3.29358	36	6	Subject 008	61.623817	2.121749	4.825674
14996	3.67505	-7.21484	625.12500	-13.25586	30.31641	-3.33411	36	6	Subject 008	61.623817	2.120513	4.849656
14997	3.41003	-6.76709	625.84375	-13.22949	30.23730	-3.40369	36	6	Subject 008	61.623817	2.113865	4.831446
14998	3.05103	-7.47876	626.53125	-13.20020	30.25684	-3.31079	36	6	Subject 008	61.623817	2.106870	4.829263
14999	3.31970	-7.12305	625.50000	-13.30273	30.30273	-3.27930	36	6	Subject 008	61.623817	2.126735	4.844561

df.sample(5)

	fx	fy	fz	mx	my	mz	Trial Number	Trial Type	Subject Number	Body Mass (Kg)	AP Sway (cm)	ML Sway (cm)
13347	-7.83765	-10.24561	791.71875	6.52124	-0.81161	-19.33398	19	1	Subject 001	79.283400	-0.823681	-0.102512
4899	-2.62964	-4.45581	786.15625	-3.81213	31.75586	-8.51221	16	4	Subject 003	72.505355	0.484907	4.039383
13386	0.14735	-7.71460	571.87500	9.97510	-13.05713	-9.95898	7	1	Subject 007	56.076363	-1.744280	-2.283214
2186	-7.97803	-10.50342	796.00000	-2.60474	4.69678	-19.92090	23	5	Subject 001	79.283400	0.327229	0.590048
5352	-2.58728	-6.48511	533.53125	30.89355	-4.84692	-9.59717	8	2	Subject 006	52.497061	-5.790392	-0.908460

df.shape

(4141622, 12)

df.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 4141622 entries, 0 to 14999
Data columns (total 12 columns):
 #   Column          Dtype  
---  ------          -----  
 0   fx              float64
 1   fy              float64
 2   fz              float64
 3   mx              float64
 4   my              float64
 5   mz              float64
 6   Trial Number    int64  
 7   Trial Type      int64  
 8   Subject Number  object 
 9   Body Mass (Kg)  float64
 10  AP Sway (cm)    float64
 11  ML Sway (cm)    float64
dtypes: float64(9), int64(2), object(1)
memory usage: 410.8+ MB

df.isnull().sum()

fx                0
fy                0
fz                0
mx                0
my                0
mz                0
Trial Number      0
Trial Type        0
Subject Number    0
Body Mass (Kg)    0
AP Sway (cm)      0
ML Sway (cm)      0
dtype: int64

df.describe()

	fx	fy	fz	mx	my	mz	Trial Number	Trial Type	Body Mass (Kg)	AP Sway (cm)	ML Sway (cm)
count	4.141622e+06	4.141622e+06	4.141622e+06	4.141622e+06	4.141622e+06	4.141622e+06	4.141622e+06	4.141622e+06	4.141622e+06	4.141622e+06	4.141622e+06
mean	-1.618832e+00	-6.067932e+00	6.225338e+02	1.045832e+00	1.982484e-01	-9.915073e+00	1.856841e+01	3.497050e+00	6.066514e+01	-2.527847e-01	-1.130701e-02
std	6.444667e+00	5.163432e+00	1.048473e+02	1.459436e+01	1.016796e+01	4.304973e+00	1.043083e+01	1.710644e+00	1.004990e+01	2.333414e+00	1.663172e+00
min	-1.455781e+02	-9.714844e+01	2.827150e+00	-8.621094e+01	-1.003633e+02	-4.004688e+01	1.000000e+00	1.000000e+00	4.815877e+01	-1.178476e+01	-3.807063e+01
25%	-3.529790e+00	-9.593750e+00	5.490625e+02	-6.791260e+00	-6.246768e+00	-1.054639e+01	1.000000e+01	2.000000e+00	5.249706e+01	-1.759027e+00	-1.050991e+00
50%	-1.272030e+00	-5.771730e+00	5.980000e+02	2.470210e+00	3.567300e-01	-9.020510e+00	1.900000e+01	3.000000e+00	6.162382e+01	-4.092785e-01	5.898343e-02
75%	1.171750e+00	-2.755620e+00	6.537188e+02	1.052832e+01	6.556400e+00	-7.652100e+00	2.800000e+01	5.000000e+00	6.281340e+01	1.095831e+00	1.056186e+00
max	1.975547e+02	8.324219e+01	9.651562e+02	6.685938e+01	6.570703e+01	4.880620e+00	3.700000e+01	6.000000e+00	7.928340e+01	3.445227e+01	2.348549e+01

df['Trial Type'].unique()

array([1, 2, 3, 4, 5, 6])

• With the unique() function, we can see the unique values in a feature. With the nunique() function, we can see the number of these unique values.

df['Trial Number'].unique()

array([ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17,
       18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
       35, 36, 37])

df['AP Sway (cm)'].describe()

count    4.141622e+06
mean    -2.527847e-01
std      2.333414e+00
min     -1.178476e+01
25%     -1.759027e+00
50%     -4.092785e-01
75%      1.095831e+00
max      3.445227e+01
Name: AP Sway (cm), dtype: float64

df['ML Sway (cm)'].describe()

count    4.141622e+06
mean    -1.130701e-02
std      1.663172e+00
min     -3.807063e+01
25%     -1.050991e+00
50%      5.898343e-02
75%      1.056186e+00
max      2.348549e+01
Name: ML Sway (cm), dtype: float64

• The corr () function has a great importance in machine learning. After determining our target feature to predict, we look at the correlation of this feature with other features. And when training our model, we take features with a corelation of more than 20% and less than 90%. This is because features with a correlation value of less than 20% have little connection with target features and do not contribute much to predictions.
• Let me explain the reason why we get over 90% with an example: For example, we want to estimate the age of an individual, we cannot train the machine by the year of birth of the individual because we have already given the answer.

• If the correlation is greater than 90%, it means that the data replicated itself in another form.

abs(df.corr()['AP Sway (cm)'].sort_values(ascending=False))

AP Sway (cm)      1.000000
Trial Type        0.359294
Body Mass (Kg)    0.267757
fz                0.215750
mz                0.134368
my                0.024728
Trial Number      0.002883
ML Sway (cm)      0.000972
fx                0.007695
fy                0.157106
mx                0.966911
Name: AP Sway (cm), dtype: float64

abs(df.corr()['ML Sway (cm)'].sort_values(ascending=False))

ML Sway (cm)      1.000000
my                0.944264
mz                0.294819
fz                0.154054
Body Mass (Kg)    0.108027
Trial Number      0.033759
AP Sway (cm)      0.000972
Trial Type        0.005355
mx                0.021619
fy                0.029860
fx                0.116864
Name: ML Sway (cm), dtype: float64

• The "Subject Number" feature is an object feature. Since we want to include it in this feature while training our model, we must make minor changes in this feature. Normally I can convert a feature with int or float values into an object feature, but I cannot convert a feature with an object (containing a string) to int directly with the astype (int) function. Therefore, we replace the "Subject 00" part, which is common in this feature with the replace () function, with nothing. Now we only have participant numbers 1 to 8 in our columns. Now we can convert this feature to an int value.

• We can convert the Subject feature from object to int without using LabelEncoding. I'll also show you the use of LabelEncoding below.

#df['Subject Number'].replace({'Subject 00':''},regex=True,inplace=True)
#df['Subject Number']=df['Subject Number'].astype(int);

abs(df.corr()['Subject Number'].sort_values(ascending=False))

Subject Number    1.000000
mz                0.649267
fx                0.175830
ML Sway (cm)      0.070002
my                0.068177
fy                0.064284
mx                0.040048
Trial Type        0.004206
Trial Number      0.006049
AP Sway (cm)      0.036516
Body Mass (Kg)    0.488195
fz                0.526974
Name: Subject Number, dtype: float64

abs(df.corr()['Trial Type'].sort_values(ascending=False))

Trial Type        1.000000
AP Sway (cm)      0.359294
Trial Number      0.154506
mz                0.060668
Body Mass (Kg)    0.005416
Subject Number    0.004206
my                0.000423
fz                0.000102
ML Sway (cm)      0.005355
fx                0.038929
fy                0.044938
mx                0.348358
Name: Trial Type, dtype: float64

abs(df.corr()['Body Mass (Kg)'].sort_values(ascending=False))

Body Mass (Kg)    1.000000
fz                0.959601
AP Sway (cm)      0.267757
ML Sway (cm)      0.108027
my                0.101717
Trial Type        0.005416
Trial Number      0.004147
fx                0.215465
fy                0.235221
mx                0.254419
Subject Number    0.488195
mz                0.638235
Name: Body Mass (Kg), dtype: float64

Pandas Profiling Report

• Thanks to Pandas profling, we can see aspects of data that we have never seen before.
  • We can see detailed examination of each feature, 4 different types of corelations, loss values and examples of different parts of the data and etc.

• Before using Pandas profing, you have to install it on your computer with (! Pip3 install pandas_profiling --upgrade).

!pip3 install pandas_profiling --upgrade

import pandas_profiling

df.profile_report()

Vizualization

• We see the corelation values of all features with each other. As the color gets lighter, the correlation between those two columns increases. In addition, with annot = True, we can see the corelation values between features inside the boxes.

fig, ax = plt.subplots(figsize=(15,15)) 
sns.heatmap(df.corr(), annot=True, linewidths=.5, ax=ax);

-In this graph, we can see the weight distribution in all data. With hue = df ['Subject Number'], we add another dimension to the graph, so we can see the weight distributions of 8 different individuals participating in the experiment. With plt.xticks (rotation = 45), we set the angle of the text on the x-axis of the chart to 45 degrees so that we can see the values more clearly.

plt.figure(figsize=(10,8))
sns.countplot(x=df['Body Mass (Kg)'],hue=df['Subject Number'],palette='hot')
plt.xticks(rotation=45);

• The purpose of using dist plot in data visualization is to see the distribution of data (such as linear or nonlinear). The purpose of using box plot is to see outlier data in the data. Because we need to clear this outlier data before doing machine learning.

fig=plt.figure(figsize=(20,20))
ax=fig.add_subplot(3,1,1)
ax=sns.boxplot(df['fx'])

bx=fig.add_subplot(3,1,2)
bx=sns.boxplot(df['fy'])

cx=fig.add_subplot(3,1,3)
cx=sns.boxplot(df['fz']);

fig=plt.figure(figsize=(20,20))
ax=fig.add_subplot(3,1,1)
ax=sns.boxplot(df['mx'])

bx=fig.add_subplot(3,1,2)
bx=sns.boxplot(df['my'])

cx=fig.add_subplot(3,1,3)
cx=sns.boxplot(df['mz']);

fig=plt.figure(figsize=(20,20))

ax=fig.add_subplot(3,1,1)
ax=sns.boxplot(df['AP Sway (cm)'],hue=df['Subject Number'])
ax.set_title('AP SWAY')

bx=fig.add_subplot(3,1,2)
bx=sns.boxplot(df['ML Sway (cm)'],hue=df['Subject Number'])
bx.set_title('ML Sway (cm)')

cx=fig.add_subplot(3,1,3)
cx=sns.boxplot(df['Body Mass (Kg)'],hue=df['Subject Number'])
cx.set_title('Body Mass (Kg)');

• On the force plate, the sways of individuals under 6 different conditions are measured and we can clearly see the rate of using these measurements in the experiment with the pie chart.

plt.figure(figsize=(15,15))
df['Trial Type'].value_counts().plot.pie(autopct='%1.1f%%',startangle=60)
plt.title('Trial Type');

plt.figure(figsize=(15,15))
df['Trial Number'].value_counts().plot.pie(autopct='%1.1f%%',startangle=60)
plt.title('Trial Number');

plt.figure(figsize=(15,15))
df['Subject Number'].value_counts().plot.pie(autopct='%1.1f%%',startangle=60)
plt.title('Subject Number');

sns.pairplot(df[['AP Sway (cm)','ML Sway (cm)','Body Mass (Kg)','Trial Number','Trial Type']]);

fig=plt.figure(figsize=(10,10))
ax=fig.add_subplot(3,1,1)
ax.hist(df['AP Sway (cm)'],bins=50, facecolor='green',alpha=0.5)
ax.set_title('AP Sway (cm)')
bx=fig.add_subplot(3,1,2)
bx.hist(df['ML Sway (cm)'],bins=50, facecolor='green', alpha=0.5)
bx.set_title('ML Sway (cm)')
cx=fig.add_subplot(3,1,3)
cx.hist(df['Body Mass (Kg)'],bins=50, facecolor='green', alpha=0.5)
cx.set_title('Body Mass (Kg)');

plt.figure(figsize=(10,10))
sns.scatterplot(x='AP Sway (cm)',y='ML Sway (cm)',hue='Trial Type',data=df,palette='hot')
plt.show();

Normalize the Data

• Normalization is a technique often applied as part of data preparation for machine learning. The goal of normalization is to change the values of numeric features in the dataset to a common scale, without distorting differences in the ranges of values.

from sklearn.preprocessing import MinMaxScaler

def normalize_(x):
    scaler=MinMaxScaler()
    return scaler.fit_transform(x)

• The difference of regression and classification: In regression, we estimate the number, but in classification, we predict the questions that have a yes-no result. (In multinominal classification, the result may be more than 2 options.)
• Therefore, we will use regression models to estimate AP and ML sway values. We use classification models to predict the athlete and what type of diving. If the data is more than 3 thousand, we can use deep learning, so we can get better results. Since our current data consists of 4 million lines, we will also use deep learning.

Regression

from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge,Lasso
from sklearn.linear_model import ElasticNet
from sklearn.neighbors import KNeighborsRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.tree import ExtraTreeRegressor
from sklearn.tree import DecisionTreeRegressor
from xgboost import XGBRegressor
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error,r2_score,mean_absolute_error

• I have included our models for Regression. I want to measure my success rate and error rate in more than one model and I don't want to write these codes separately for each model. For this reason, I am writing a regression function, after that I will determine my goal and the columns I will train, send it to the function and see the results in a table.

def algo_test(x,y):

        #I include the regression models I will use.
        L=LinearRegression()
        R=Ridge()
        Lass=Lasso()
        E=ElasticNet()
        ETR=ExtraTreeRegressor()
        GBR=GradientBoostingRegressor()
        XGBC=XGBRegressor()
        dt=DecisionTreeRegressor()
        kn=KNeighborsRegressor()
        
        #I separate the data as train and test. (20% test)
        x_train,x_test,y_train,y_test=train_test_split(x,y,test_size=.2,random_state=42)
        
        #I put my models and model names in a list.
        algos=[L,R,Lass,E,ETR,GBR,XGBC,dt,kn]
        algo_names=['Linear','Ridge','Lasso','ElasticNet','Extra Tree','Gradient Boosting','XGradientBoosting','DecisionTree','KNeighbors']
        
        #I create empty lists for error rates and success rates, which I will fill in later.
        r2Score = []
        rmse = []
        mae = []
        
        #I am creating a dataframe as I want to see all results as a table. 
        #Its columns will be 'R2_score', 'RMSE', 'MAE'. The indexes will take from the string I created for the model names.
        result=pd.DataFrame(columns=['R2_score','RMSE','MAE'],index=algo_names)
        
        #I run the regression algorithms for each model and find the results of r2_score, mean_absolute_error and mean_squared_error.
        #And I add these results I found to the empty lists I created above with the append () function.
        for algo in algos:
            p=algo.fit(x_train,y_train).predict(x_test)
            r2Score.append(r2_score(y_test,p))
            rmse.append(mean_squared_error(y_test,p)**.5)
            mae.append(mean_absolute_error(y_test,p))
        
        #I fill the columns in the table with these lists.    
        result['R2_score']=r2Score 
        result.RMSE=rmse
        result.MAE=mae

        #I sort my result table by r2_score value and return it.
        return result.sort_values('R2_score',ascending=False)

df.sample(5)

	fx	fy	fz	mx	my	mz	Trial Number	Trial Type	Subject Number	Body Mass (Kg)	AP Sway (cm)	ML Sway (cm)
2029	-5.63696	-10.12744	777.37500	-3.76953	-11.71631	-21.52539	2	2	1	79.283400	0.484905	-1.507163
12358	-4.18677	-8.00293	496.01563	2.67285	-10.31836	-9.90381	17	5	5	48.158771	-0.538864	-2.080249
82	-10.47607	-5.76074	780.93750	-23.30371	-16.53906	-19.67285	3	3	1	79.283400	2.984069	-2.117847
6992	0.81354	-5.01855	504.76563	-9.07129	4.46216	-7.35498	12	6	5	48.158771	1.797129	0.884006
10832	-1.21808	-5.42065	554.15625	6.98145	3.46460	-7.66089	4	4	2	52.299497	-1.259834	0.625203

• We predict the AP sway values by giving the force plate measurements of the athletes. And we can make this prediction with the KNeighbors model with a success rate of 0.993920.

• Y is always given the target feature, and this feature from x is dropped with the drop () function.

y=df['AP Sway (cm)']
x=df.drop(columns=['AP Sway (cm)','mx'],axis=1)

x=pd.get_dummies(x,drop_first=True)

• While specifying x, we drop the target feature and mx feature because corr is above 90%.

• Before sending x and y to the algo_test function, we normalize x with the normalize function. This process increases the success rate. (We do not normalize the target feature.)

x=normalize_(x)

algo_test(x,y)

	R2_score	RMSE	MAE
KNeighbors	0.993920	0.181741	0.086295
Extra Tree	0.978061	0.345217	0.155069
DecisionTree	0.977649	0.348446	0.148874
XGradientBoosting	0.927642	0.626944	0.456992
Gradient Boosting	0.751565	1.161692	0.898013
Linear	0.507610	1.635458	1.264625
Ridge	0.507608	1.635461	1.264665
Lasso	-0.000009	2.330702	1.794591
ElasticNet	-0.000009	2.330702	1.794591

• We predict the ML sway values by giving the force plate measurements of the athletes.

y=df['ML Sway (cm)']
x=df.drop(columns=['ML Sway (cm)','my'],axis=1)

x=pd.get_dummies(x,drop_first=True)

x=normalize_(x)

algo_test(x,y)

	R2_score	RMSE	MAE
KNeighbors	9.943916e-01	0.124316	0.053447
Extra Tree	9.756195e-01	0.259195	0.105745
DecisionTree	9.731581e-01	0.271965	0.104258
XGradientBoosting	9.206857e-01	0.467500	0.309166
Gradient Boosting	7.729252e-01	0.791025	0.561445
Ridge	4.260412e-01	1.257611	0.942964
Linear	4.260342e-01	1.257618	0.943011
Lasso	-2.423055e-10	1.659991	1.267697
ElasticNet	-2.423055e-10	1.659991	1.267697

Deep Learning

• I will use Deep Learning, one of the newest technologies of recent times, to predict ML Sway.

• We want to estimate the ML sway values by giving the force plate measurements of the athletes, but when we remove the features where corr is more than 90%, our success rate decreases, so we will try to increase the accuracy of our predictions by applying deep learning.

y=data[['ML Sway (cm)']]
x=data.drop(columns=['ML Sway (cm)','my'],axis=1)

x=pd.get_dummies(x,drop_first=True)

x=normalize_(x)

x_train,x_test,y_train,y_test=train_test_split(x,y,random_state=42)

• Ways to increase success in Deep Learning:
  • The number of nodes in the layers can be increased
  • The number of layers can be increased
  • Dropout can be used (Dropout prevents it from memorizing the data, discards some of the data)
  • BatchNormalize () can be used (We normalize the data with BatchNormalization () after each layer.)

• If we were classifying with deep learning, the last layer would also have activation = 'sigmoid', loss = 'sparse_categorical_crossentropy' or loss = 'binary_crossentropy' and metrics = ['accuracy'].

!pip install tensorflow

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation
from tensorflow.keras.callbacks import EarlyStopping

model = Sequential()
model.add(Dense(25, input_dim=x.shape[1], activation='relu')) # Hidden 1
model.add(Dense(50, activation='relu')) # Hidden 2
model.add(Dense(50, activation='relu'))
model.add(Dense(10, activation='relu'))
model.add(Dense(1)) # Output
model.compile(loss='mean_squared_error', optimizer='adam')
monitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, 
                        patience=5, verbose=1, mode='auto', 
                        restore_best_weights=True)
model.fit(x_train,y_train,validation_data=(x_test,y_test),
          callbacks=[monitor],verbose=2,epochs=1000)

Epoch 1/1000
4688/4688 - 8s - loss: 1.1561 - val_loss: 0.6529
Epoch 2/1000
4688/4688 - 7s - loss: 0.5825 - val_loss: 0.6621
Epoch 3/1000
4688/4688 - 7s - loss: 0.5019 - val_loss: 0.5511
Epoch 4/1000
4688/4688 - 7s - loss: 0.4591 - val_loss: 0.4030
Epoch 5/1000
4688/4688 - 7s - loss: 0.4280 - val_loss: 0.3915
Epoch 6/1000
4688/4688 - 7s - loss: 0.4049 - val_loss: 0.3753
Epoch 7/1000
4688/4688 - 8s - loss: 0.3899 - val_loss: 0.3495
Epoch 8/1000
4688/4688 - 7s - loss: 0.3793 - val_loss: 0.3966
Epoch 9/1000
4688/4688 - 7s - loss: 0.3703 - val_loss: 0.3492
Epoch 10/1000
4688/4688 - 7s - loss: 0.3628 - val_loss: 0.3862
Epoch 11/1000
4688/4688 - 8s - loss: 0.3569 - val_loss: 0.3546
Epoch 12/1000
4688/4688 - 7s - loss: 0.3511 - val_loss: 0.3620
Restoring model weights from the end of the best epoch.
Epoch 00012: early stopping

<tensorflow.python.keras.callbacks.History at 0x7f6ec3cebc50>

from sklearn import metrics

# Predict
pred = model.predict(x_test)

# Measure MSE error.  
score = metrics.mean_squared_error(pred,y_test)
print("Final score (MSE): {}".format(score))

Final score (MSE): 0.3494503767025284

r2_score(y_test,pred)

0.8718408485407947

import tensorflow as tf 
model = tf.keras.Sequential([
                             tf.keras.layers.Dense(512, activation='relu'),
                             tf.keras.layers.Dense(512, activation='relu'),
                             tf.keras.layers.Dropout(0.25),
                             tf.keras.layers.Flatten(),
                             tf.keras.layers.Dense(512, activation='relu'),
                             tf.keras.layers.Dropout(0.50),
                             tf.keras.layers.Dense(512, activation='relu'),
                             tf.keras.layers.Dense(512, activation='softmax'),])

model.compile(optimizer='adam', loss='mse')
history = model.fit(x_train,y_train, batch_size=128, epochs=40, verbose=1, validation_data=(x_test,y_test))

Epoch 1/40
1172/1172 [==============================] - 8s 6ms/step - loss: 2.7225 - val_loss: 2.7580
Epoch 2/40
1172/1172 [==============================] - 8s 7ms/step - loss: 2.7212 - val_loss: 2.7580
Epoch 3/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7004 - val_loss: 2.7580
Epoch 4/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7617 - val_loss: 2.7580
Epoch 5/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7164 - val_loss: 2.7580
Epoch 6/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7224 - val_loss: 2.7580
Epoch 7/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7553 - val_loss: 2.7580
Epoch 8/40
1172/1172 [==============================] - 8s 7ms/step - loss: 2.7111 - val_loss: 2.7580
Epoch 9/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7346 - val_loss: 2.7580
Epoch 10/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7654 - val_loss: 2.7580
Epoch 11/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7319 - val_loss: 2.7580
Epoch 12/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7161 - val_loss: 2.7580
Epoch 13/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7643 - val_loss: 2.7580
Epoch 14/40
1172/1172 [==============================] - 8s 7ms/step - loss: 2.7490 - val_loss: 2.7580
Epoch 15/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7888 - val_loss: 2.7580
Epoch 16/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.6865 - val_loss: 2.7580
Epoch 17/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7348 - val_loss: 2.7580
Epoch 18/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7267 - val_loss: 2.7580
Epoch 19/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.6882 - val_loss: 2.7580
Epoch 20/40
1172/1172 [==============================] - 8s 7ms/step - loss: 2.7567 - val_loss: 2.7580
Epoch 21/40
1172/1172 [==============================] - 8s 6ms/step - loss: 2.7516 - val_loss: 2.7580
Epoch 22/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7205 - val_loss: 2.7580
Epoch 23/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7716 - val_loss: 2.7580
Epoch 24/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7432 - val_loss: 2.7580
Epoch 25/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7257 - val_loss: 2.7580
Epoch 26/40
1172/1172 [==============================] - 8s 7ms/step - loss: 2.7184 - val_loss: 2.7580
Epoch 27/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7409 - val_loss: 2.7580
Epoch 28/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7755 - val_loss: 2.7580
Epoch 29/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7129 - val_loss: 2.7580
Epoch 30/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7353 - val_loss: 2.7580
Epoch 31/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7578 - val_loss: 2.7580
Epoch 32/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7213 - val_loss: 2.7580
Epoch 33/40
1172/1172 [==============================] - 8s 7ms/step - loss: 2.7245 - val_loss: 2.7580
Epoch 34/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7550 - val_loss: 2.7580
Epoch 35/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7532 - val_loss: 2.7580
Epoch 36/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7005 - val_loss: 2.7580
Epoch 37/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7301 - val_loss: 2.7580
Epoch 38/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7095 - val_loss: 2.7580
Epoch 39/40
1172/1172 [==============================] - 8s 7ms/step - loss: 2.6976 - val_loss: 2.7580
Epoch 40/40
1172/1172 [==============================] - 7s 6ms/step - loss: 2.7206 - val_loss: 2.7580

model.summary()

Model: "sequential_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
dense_5 (Dense)              (None, 512)               5632      
_________________________________________________________________
dense_6 (Dense)              (None, 512)               262656    
_________________________________________________________________
dropout_2 (Dropout)          (None, 512)               0         
_________________________________________________________________
flatten_1 (Flatten)          (None, 512)               0         
_________________________________________________________________
dense_7 (Dense)              (None, 512)               262656    
_________________________________________________________________
dropout_3 (Dropout)          (None, 512)               0         
_________________________________________________________________
dense_8 (Dense)              (None, 512)               262656    
_________________________________________________________________
dense_9 (Dense)              (None, 512)               262656    
=================================================================
Total params: 1,056,256
Trainable params: 1,056,256
Non-trainable params: 0
_________________________________________________________________

loss_df=pd.DataFrame(model.history.history)
loss_df.plot(figsize=(12,8));

pred=model.predict(x_test)

mean_squared_error(y_test,pred)**.5

r2_score(y_test,pred)

Classification

• In ML models we are often required to convert the categorical i.e text features to its numeric representation. The two most common ways to do this is to use Label Encoder or OneHot Encoder.

• With the Unique () function, we see the initial version of the "Subject Number" column in the object state and after the LabelEncoding is applied.

df['Subject Number'].unique()

array(['Subject 001', 'Subject 002', 'Subject 003', 'Subject 004',
       'Subject 005', 'Subject 006', 'Subject 007', 'Subject 008'],
      dtype=object)

from sklearn.preprocessing import LabelEncoder
le=LabelEncoder()
df['Subject Number']=le.fit_transform(df['Subject Number'])

df['Subject Number'].unique()

array([0, 1, 2, 3, 4, 5, 6, 7])

#If I were waiting for my prediction result with 2 options as yes or no with Classification, 
#I would use both of these models, but here I will guess which of the 8 different answers.

#from sklearn.naive_bayes import GaussianNB
#from sklearn.naive_bayes import BernoulliNB

from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import GradientBoostingClassifier
from xgboost import XGBClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.naive_bayes import MultinomialNB

from sklearn.metrics import accuracy_score,precision_score,recall_score,f1_score
from sklearn.metrics import confusion_matrix,classification_report

def classification_(x,y):
    
    
    #g=GaussianNB()
    #b=BernoulliNB()
    k=KNeighborsClassifier()
    svc=SVC()
    d=DecisionTreeClassifier()
    log=LogisticRegression()
    gbc=GradientBoostingClassifier()
    mn=MultinomialNB()
    rf=RandomForestClassifier()
    ab=AdaBoostClassifier()
    
    algos=[k,svc,d,log,gbc,mn,rf,ab]
    algos_name=['KNeigbors','SVC','DecisionTree','LogisticRegr','GradientBoosting','Multinominal','RandomForest','AdaBoost']
    
    accuracy = []
    precision = []
    recall = []
    f1 = []
   
    result=pd.DataFrame(columns=['AccuracyScore','PrecisionScore','RecallScore','f1_Score'],index=algos_name)

    x_train,x_test,y_train,y_test=train_test_split(x,y,random_state=42)

    for i in algos:
        
        predict=i.fit(x_train,y_train).predict(x_test)
        
        accuracy.append(accuracy_score(y_test,predict))
        precision.append(precision_score(y_test,predict,average='weighted'))
        recall.append(recall_score(y_test,predict,average='weighted'))
        f1.append(f1_score(y_test,predict,average='weighted'))
      
    result.AccuracyScore=accuracy
    result.PrecisionScore=precision
    result.RecallScore=recall
    result.f1_Score=f1
    
    
    return result.sort_values('f1_Score',ascending=False)
    
    

• Based on the force plate data, we will predict which swimmer this data belongs to.

#train,test=train_test_split(df,test_size=.10,random_state=42)

data=df.sample(200000) #I randomly took two hundred thousand lines of data.

y=data[['Subject Number']]
x=data.drop('Subject Number',axis=1)

#y=y.astype (np.int8)

x=pd.get_dummies(x,drop_first=True)

x=normalize_(x)

classification_(x,y) 

	AccuracyScore	PrecisionScore	RecallScore	f1_Score
DecisionTree	1.00000	1.000000	1.00000	1.000000
GradientBoosting	1.00000	1.000000	1.00000	1.000000
RandomForest	1.00000	1.000000	1.00000	1.000000
KNeigbors	0.99896	0.998961	0.99896	0.998960
SVC	0.99714	0.997146	0.99714	0.997140
LogisticRegr	0.96914	0.969228	0.96914	0.969149
Multinominal	0.79848	0.819915	0.79848	0.797869
AdaBoost	0.50222	0.344426	0.50222	0.384649

from pycaret.classification import *
exp_name=setup(data=data,target='Subject Number')

	Description	Value
0	session_id	7469
1	Target	Subject Number
2	Target Type	Multiclass
3	Label Encoded	0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6, 7: 7
4	Original Data	(200000, 12)
5	Missing Values	False
6	Numeric Features	10
7	Categorical Features	1
8	Ordinal Features	False
9	High Cardinality Features	False
10	High Cardinality Method	None
11	Transformed Train Set	(139999, 16)
12	Transformed Test Set	(60001, 16)
13	Shuffle Train-Test	True
14	Stratify Train-Test	False
15	Fold Generator	StratifiedKFold
16	Fold Number	10
17	CPU Jobs	-1
18	Use GPU	False
19	Log Experiment	False
20	Experiment Name	clf-default-name
21	USI	6c3a
22	Imputation Type	simple
23	Iterative Imputation Iteration	None
24	Numeric Imputer	mean
25	Iterative Imputation Numeric Model	None
26	Categorical Imputer	constant
27	Iterative Imputation Categorical Model	None
28	Unknown Categoricals Handling	least_frequent
29	Normalize	False
30	Normalize Method	None
31	Transformation	False
32	Transformation Method	None
33	PCA	False
34	PCA Method	None
35	PCA Components	None
36	Ignore Low Variance	False
37	Combine Rare Levels	False
38	Rare Level Threshold	None
39	Numeric Binning	False
40	Remove Outliers	False
41	Outliers Threshold	None
42	Remove Multicollinearity	False
43	Multicollinearity Threshold	None
44	Clustering	False
45	Clustering Iteration	None
46	Polynomial Features	False
47	Polynomial Degree	None
48	Trignometry Features	False
49	Polynomial Threshold	None
50	Group Features	False
51	Feature Selection	False
52	Features Selection Threshold	None
53	Feature Interaction	False
54	Feature Ratio	False
55	Interaction Threshold	None
56	Fix Imbalance	False
57	Fix Imbalance Method	SMOTE

best_model=compare_models()

	Model	Accuracy	AUC	Recall	Prec.	F1	Kappa	MCC	TT (Sec)
nb	Naive Bayes	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	0.1210
dt	Decision Tree Classifier	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	0.0840
rf	Random Forest Classifier	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	1.7960
gbc	Gradient Boosting Classifier	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	31.8440
et	Extra Trees Classifier	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	1.1390
xgboost	Extreme Gradient Boosting	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	53.7110
lightgbm	Light Gradient Boosting Machine	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	1.2190
catboost	CatBoost Classifier	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	22.2410
knn	K Neighbors Classifier	0.9915	0.9987	0.9915	0.9915	0.9915	0.9903	0.9903	0.4310
lda	Linear Discriminant Analysis	0.9128	0.9894	0.9122	0.9140	0.9131	0.9004	0.9005	0.1020
lr	Logistic Regression	0.7869	0.9598	0.7853	0.7863	0.7841	0.7564	0.7572	6.1650
qda	Quadratic Discriminant Analysis	0.7837	0.8764	0.7840	0.8825	0.7812	0.7528	0.7730	0.0430
svm	SVM - Linear Kernel	0.7609	0.0000	0.7600	0.8315	0.7230	0.7267	0.7470	2.9400
ridge	Ridge Classifier	0.7501	0.0000	0.7495	0.7678	0.7402	0.7143	0.7188	0.0340
ada	Ada Boost Classifier	0.4551	0.7977	0.4500	0.3404	0.3616	0.3756	0.4838	1.0460

• Given the sways of the athletes, we will predict which of the 8 types of movement they do with classification.

y=data['Trial Type']
x=data.drop('Trial Type',axis=1)

x=pd.get_dummies(x,drop_first=True)

x=normalize_(x)

classification_(x,y)

	AccuracyScore	PrecisionScore	RecallScore	f1_Score
GradientBoosting	1.00000	1.000000	1.00000	1.000000
DecisionTree	0.99996	0.999960	0.99996	0.999960
KNeigbors	0.99524	0.995237	0.99524	0.995235
RandomForest	0.99224	0.992240	0.99224	0.992239
AdaBoost	0.89394	0.905268	0.89394	0.893524
SVC	0.59808	0.611244	0.59808	0.595301
LogisticRegr	0.32274	0.320847	0.32274	0.320060
Multinominal	0.22202	0.252846	0.22202	0.157875

from pycaret.classification import *
exp_name=setup(data=data,target='Trial Type')

	Description	Value
0	session_id	298
1	Target	Trial Type
2	Target Type	Multiclass
3	Label Encoded	1: 0, 2: 1, 3: 2, 4: 3, 5: 4, 6: 5
4	Original Data	(200000, 12)
5	Missing Values	False
6	Numeric Features	10
7	Categorical Features	1
8	Ordinal Features	False
9	High Cardinality Features	False
10	High Cardinality Method	None
11	Transformed Train Set	(139999, 18)
12	Transformed Test Set	(60001, 18)
13	Shuffle Train-Test	True
14	Stratify Train-Test	False
15	Fold Generator	StratifiedKFold
16	Fold Number	10
17	CPU Jobs	-1
18	Use GPU	False
19	Log Experiment	False
20	Experiment Name	clf-default-name
21	USI	2b23
22	Imputation Type	simple
23	Iterative Imputation Iteration	None
24	Numeric Imputer	mean
25	Iterative Imputation Numeric Model	None
26	Categorical Imputer	constant
27	Iterative Imputation Categorical Model	None
28	Unknown Categoricals Handling	least_frequent
29	Normalize	False
30	Normalize Method	None
31	Transformation	False
32	Transformation Method	None
33	PCA	False
34	PCA Method	None
35	PCA Components	None
36	Ignore Low Variance	False
37	Combine Rare Levels	False
38	Rare Level Threshold	None
39	Numeric Binning	False
40	Remove Outliers	False
41	Outliers Threshold	None
42	Remove Multicollinearity	False
43	Multicollinearity Threshold	None
44	Clustering	False
45	Clustering Iteration	None
46	Polynomial Features	False
47	Polynomial Degree	None
48	Trignometry Features	False
49	Polynomial Threshold	None
50	Group Features	False
51	Feature Selection	False
52	Features Selection Threshold	None
53	Feature Interaction	False
54	Feature Ratio	False
55	Interaction Threshold	None
56	Fix Imbalance	False
57	Fix Imbalance Method	SMOTE

best_model=compare_models()

	Model	Accuracy	AUC	Recall	Prec.	F1	Kappa	MCC	TT (Sec)
gbc	Gradient Boosting Classifier	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	24.2340
xgboost	Extreme Gradient Boosting	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	120.8430
lightgbm	Light Gradient Boosting Machine	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	1.4350
catboost	CatBoost Classifier	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	1.0000	17.1080
dt	Decision Tree Classifier	0.9993	0.9996	0.9993	0.9993	0.9993	0.9992	0.9992	0.2420
et	Extra Trees Classifier	0.9986	1.0000	0.9986	0.9986	0.9986	0.9983	0.9983	1.4900
rf	Random Forest Classifier	0.9860	0.9996	0.9860	0.9860	0.9860	0.9832	0.9832	2.7540
knn	K Neighbors Classifier	0.9163	0.9888	0.9162	0.9166	0.9155	0.8996	0.8999	0.4370
ada	Ada Boost Classifier	0.8927	0.9475	0.8928	0.9012	0.8923	0.8713	0.8732	1.0070
lr	Logistic Regression	0.3489	0.7437	0.3485	0.3468	0.3457	0.2185	0.2190	5.2200
lda	Linear Discriminant Analysis	0.3429	0.7428	0.3426	0.3403	0.3409	0.2113	0.2115	0.0670
ridge	Ridge Classifier	0.3349	0.0000	0.3344	0.3281	0.3233	0.2017	0.2038	0.0320
nb	Naive Bayes	0.3149	0.7270	0.3148	0.3339	0.3141	0.1778	0.1796	0.1250
svm	SVM - Linear Kernel	0.2575	0.0000	0.2578	0.2638	0.1689	0.1093	0.1433	5.0160
qda	Quadratic Discriminant Analysis	0.1664	0.4998	0.1664	0.1164	0.1268	-0.0004	-0.0004	0.0400

Clustering

• By clustering, we can divide the swimmers participating in the experiment into groups with the elbow method.

from sklearn.metrics import silhouette_score
from sklearn.cluster import KMeans
def cluster(x,clusters):

    allscore = []
    allclusters = []
    sum_of_squared_distances = []
    
    for i in np.arange(1,clusters):
        
        i+=1
        model=KMeans(n_clusters=i)
        pred=model.fit_predict(x)
        s_score=silhouette_score(x,pred)
        print('Number of cluster: {} , Score: {}'.format(i,s_score))
        allscore.append(s_score)
        allclusters.append(i)
        sum_of_squared_distances.append(model.inertia_)
        
    plt.figure(figsize=(10,10))
    plt.plot(allclusters,sum_of_squared_distances,marker='x',c='r');
    

• I choose those features according to which characteristics I want to group the athletes.

data=data[['AP Sway (cm)','ML Sway (cm)','Body Mass (Kg)','fx','fy','fx','mx','my','mz','Trial Type','Trial Number']]

cluster(data,10)

Number of cluster: 2 , Score: 0.2480354202568269
Number of cluster: 3 , Score: 0.16780576920769105
Number of cluster: 4 , Score: 0.19101796204408872
Number of cluster: 5 , Score: 0.17448091098147364
Number of cluster: 6 , Score: 0.17678157079048742
Number of cluster: 7 , Score: 0.1855067350533299
Number of cluster: 8 , Score: 0.18188846763237837
Number of cluster: 9 , Score: 0.1722050295786211
Number of cluster: 10 , Score: 0.17613614799498392

model=KMeans(n_clusters=4)
pred=model.fit_predict(data)
silhouette_score(data,pred)

0.19088393397270448

data['Cluster']=pred

data.sample(10)

	AP Sway (cm)	ML Sway (cm)	Body Mass (Kg)	fx	fy	fx	mx	my	mz	Trial Type	Trial Number	Cluster
1512	2.303066	3.770239	61.623817	-0.76657	-4.38306	-0.76657	-14.17969	23.21289	-3.41309	4	34	2
8276	2.579357	-0.856807	72.505355	-6.80957	-4.38086	-6.80957	-20.08594	-6.67212	-13.30762	5	35	2
13818	-1.809796	-0.373419	62.813397	3.07947	1.73547	3.07947	11.60645	-2.39478	-9.22314	2	32	0
13344	-1.961598	1.070611	52.497061	-3.78650	-12.64746	-3.78650	10.76611	5.87598	-8.51855	2	32	0
6206	1.314207	0.933026	72.505355	0.61298	-5.73804	0.61298	-10.24219	7.27148	-9.31006	1	7	2
9820	1.131494	-2.755242	72.505355	4.88354	-2.86462	4.88354	-8.77686	-21.37207	-10.70020	3	21	2
10311	-0.004262	0.632416	62.813397	-2.46741	-4.34839	-2.46741	0.02702	4.00952	-9.50928	2	2	3
11095	-1.170002	0.655741	79.283400	-8.18311	-12.23340	-8.18311	9.29932	5.21191	-18.19531	1	13	1
8142	-5.374999	-1.156022	52.497061	-2.99719	-8.25879	-2.99719	28.68066	-6.16846	-9.61279	2	8	3
786	-0.200660	1.513623	52.497061	-5.21436	-9.22119	-5.21436	1.10852	8.36182	-7.92261	4	34	0

• With pycaret clustering, we don't need to write the function above. pycaret makes the most appropriate grouping according to the elbow method.

!pip install pycaret

data=data.reset_index(drop=True)

#data=data.drop('index',axis=1)

from pycaret.clustering import *
clul=setup(data,session_id=123,log_experiment=True,log_plots=True,
          experiment_name='Swimmer Clustering')

kmeans=create_model('kmeans',num_clusters=4)

plot_model(kmeans)

plot_model(kmeans,plot='elbow')

plot_model(kmeans,feature='AP Sway (cm)',label=True)

plot_model(kmeans,feature='ML Sway (cm)',label=True)

plot_model(kmeans,plot='tsne')